from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re

print "Begins..."
BIG_NUMBER = 100000000
# termID
# firstFactorProbablity
firstFactorProbablityDict = {}
inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/data/LMs/GOV2/wholeLexicon_GOV2_goodTuring_unigramProbablity_20140619_SIMPLIFIED"
inputFileHandler = open(inputFileName0,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    if lineElements[0] != "UNK":
        termID = int(lineElements[0])
    else:
        termID = BIG_NUMBER
    firstFactorProbability = float(lineElements[1])
    if termID not in firstFactorProbablityDict:
        firstFactorProbablityDict[termID] = firstFactorProbability 
inputFileHandler.close()
print "len(firstFactorProbablityDict):",len(firstFactorProbablityDict)
print "firstFactorProbablityDict[33254377]:",firstFactorProbablityDict[33254377]

# 4 files are needed here
inputFileList = []
# inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/dynamicUnigramFromWei/LEAVE_documentPostingArray_clueweb09B_WHOLE_20140613.binary"
inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/data/Gov2_document_posting_arrays/LEAVE_selectedDocumentPostingValuesInfo_0_6M_DOC_20140518_TOP100Postings.binary"
inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/data/Gov2_document_posting_arrays/LEAVE_selectedDocumentPostingValuesInfo_6M_12M_DOC_20140518_TOP100Postings.binary"
inputFileName3 = "/local_scratch/wei/workspace/NYU_IRTK/data/Gov2_document_posting_arrays/LEAVE_selectedDocumentPostingValuesInfo_12M_18Dot5M_DOC_20140518_TOP100Postings.binary"
inputFileName4 = "/local_scratch/wei/workspace/NYU_IRTK/data/Gov2_document_posting_arrays/LEAVE_selectedDocumentPostingValuesInfo_18Dot5M_25M_DOC_20140518_TOP100Postings.binary"
# inputFileList.append(inputFileName1)
# inputFileList.append(inputFileName2)
# inputFileList.append(inputFileName3)
inputFileList.append(inputFileName4)

for currentInputFileName in inputFileList:
    print "currentInputFileName:",currentInputFileName
    inputFileHandler = open(currentInputFileName,"rb")
    statinfo = os.stat(currentInputFileName)
    fileSize = statinfo.st_size
    print "file size:",fileSize
    numOfBytesRead = 0
    while numOfBytesRead < fileSize:
        byteString = inputFileHandler.read(4 + 4)
        (docID,numOfPostings) = unpack( "2I", byteString)
        xDoc = 0.0
        sumStatic0 = 0.0
        sumStatic1 = 0.0
        sumStatic2 = 0.0
        sumStatic3 = 0.0
        sumStatic4 = 0.0
        sumScore = 0.0
        for i in range(0,numOfPostings):
            byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
            (termID,static,_,_,score) = unpack( "1I4f", byteString)
            # print "----->",termID,static,score
            firstFactorProbablity = 0.0
            secondFactorProbablity = 0.0
            secondFactorProbablityPOWTo0 = 0.0
            secondFactorProbablityPOWTo1 = 0.0
            secondFactorProbablityPOWTo2 = 0.0
            secondFactorProbablityPOWTo3 = 0.0
            secondFactorProbablityPOWTo4 = 0.0
            newStatic0 = 0.0
            newStatic1 = 0.0
            newStatic2 = 0.0
            newStatic3 = 0.0
            newStatic4 = 0.0
            if termID in firstFactorProbablityDict:
                firstFactorProbablity = firstFactorProbablityDict[termID]
            else:
                firstFactorProbablity = firstFactorProbablityDict[BIG_NUMBER]
            secondFactorProbablity = static / firstFactorProbablity
            secondFactorProbablityPOWTo0 = math.pow(secondFactorProbablity, 0)
            secondFactorProbablityPOWTo1 = math.pow(secondFactorProbablity, 1)
            secondFactorProbablityPOWTo2 = math.pow(secondFactorProbablity, 0.3)
            secondFactorProbablityPOWTo3 = math.pow(secondFactorProbablity, 0.5)
            secondFactorProbablityPOWTo4 = math.pow(secondFactorProbablity, 0.7)
            newStatic0 = firstFactorProbablity * secondFactorProbablityPOWTo0
            newStatic1 = firstFactorProbablity * secondFactorProbablityPOWTo1
            newStatic2 = firstFactorProbablity * secondFactorProbablityPOWTo2
            newStatic3 = firstFactorProbablity * secondFactorProbablityPOWTo3
            newStatic4 = firstFactorProbablity * secondFactorProbablityPOWTo4
            # print termID,firstFactorProbablity,secondFactorProbablity,newStatic0,newStatic1,newStatic2,newStatic3,newStatic4,score
            xDoc += firstFactorProbablity
            sumStatic0 += newStatic0
            sumStatic1 += newStatic1
            sumStatic2 += newStatic2
            sumStatic3 += newStatic3
            sumStatic4 += newStatic4
            sumScore += score
        numOfBytesRead += 8 + numOfPostings * 4 * 5
        # inputFileHandler.seek(numOfBytesRead)
        print docID,numOfPostings,xDoc,sumStatic0,sumStatic1,sumStatic2,sumStatic3,sumStatic4,sumScore
        # if docID == 1:
        #    exit(1)
    inputFileHandler.close()
print "Ends."







